{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='svg'\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt  # Plotting library\n",
    "from matplotlib.colors import colorConverter, ListedColormap\n",
    "from matplotlib import cm # Colormaps\n",
    "import seaborn as sns  # Fancier plots\n",
    "\n",
    "# Set matplotlib and seaborn plotting style\n",
    "sns.set_style('darkgrid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Logistic Regression 不能解决 XOR 问题\n",
    "Copyleft Ruiming Guo\n",
    "All wrongs reversed.\n",
    "\"\"\"\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "\n",
    "\n",
    "x_data = torch.tensor([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=torch.float32)\n",
    "y_data = torch.tensor([[0], [1], [1], [0]])\n",
    "\n",
    "class LogisticRegressionModel(nn.Module):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "        self.W = nn.Parameter(torch.randn((2, 1)))\n",
    "        self.b = nn.Parameter(torch.randn((1,)))\n",
    "\n",
    "    def forward(self, X: torch.Tensor) -> torch.Tensor:\n",
    "        return torch.sigmoid(X @ self.W + self.b)\n",
    "\n",
    "class Cost(nn.Module):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, hypothesis: torch.Tensor, Y: torch.Tensor) -> torch.Tensor:\n",
    "        return -torch.mean(Y * torch.log(hypothesis) + (1-Y) * torch.log(1-hypothesis))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def accuracy(hypothesis: torch.FloatTensor, Y: torch.Tensor) -> torch.Tensor:\n",
    "    return ((hypothesis > 0.5).long() == Y.long()).sum() / Y.shape[0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step=0, loss=0.779062807559967, accuracy=0.75\n",
      "Hypothesis:  tensor([[0.3759],\n",
      "        [0.5222],\n",
      "        [0.1963],\n",
      "        [0.3071]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=2000, loss=0.6931471824645996, accuracy=0.25\n",
      "Hypothesis:  tensor([[0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=4000, loss=0.6931471824645996, accuracy=0.5\n",
      "Hypothesis:  tensor([[0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=6000, loss=0.6931471824645996, accuracy=0.5\n",
      "Hypothesis:  tensor([[0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=8000, loss=0.6931471824645996, accuracy=0.5\n",
      "Hypothesis:  tensor([[0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000],\n",
      "        [0.5000]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegressionModel()\n",
    "cost = Cost()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "model.to(device)\n",
    "cost.to(device)\n",
    "x_data = x_data.to(device)\n",
    "y_data = y_data.to(device)\n",
    "\n",
    "loss_record = []\n",
    "acc_record = []\n",
    "\n",
    "for step in range(10000):\n",
    "    hypot = model(x_data)\n",
    "    loss = cost(hypot, y_data)\n",
    "    loss_record.append(float(loss))\n",
    "    acc = accuracy(hypot, y_data)\n",
    "    acc_record.append(float(acc))\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if step % 2000 == 0:\n",
    "\n",
    "        print(f\"step={step}, loss={float(loss)}, accuracy={float(acc)}\")\n",
    "        print(\"Hypothesis: \", hypot)\n",
    "        print(\"Correct: \", y_data)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fc868766550>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"372.821633pt\" height=\"250.256431pt\" viewBox=\"0 0 372.821633 250.256431\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n <metadata>\n  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n   <cc:Work>\n    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n    <dc:date>2022-07-13T09:37:01.105621</dc:date>\n    <dc:format>image/svg+xml</dc:format>\n    <dc:creator>\n     <cc:Agent>\n      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n     </cc:Agent>\n    </dc:creator>\n   </cc:Work>\n  </rdf:RDF>\n </metadata>\n <defs>\n  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 250.256431 \nL 372.821633 250.256431 \nL 372.821633 0 \nL 0 0 \nz\n\" style=\"fill: #ffffff\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 30.103125 226.378306 \nL 364.903125 226.378306 \nL 364.903125 8.938306 \nL 30.103125 8.938306 \nz\n\" style=\"fill: #eaeaf2\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <path d=\"M 45.321307 226.378306 \nL 45.321307 8.938306 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <g style=\"fill: #262626\" transform=\"translate(42.140057 240.976743)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \nQ 1547 4250 1301 3770 \nQ 1056 3291 1056 2328 \nQ 1056 1369 1301 889 \nQ 1547 409 2034 409 \nQ 2525 409 2770 889 \nQ 3016 1369 3016 2328 \nQ 3016 3291 2770 3770 \nQ 2525 4250 2034 4250 \nz\nM 2034 4750 \nQ 2819 4750 3233 4129 \nQ 3647 3509 3647 2328 \nQ 3647 1150 3233 529 \nQ 2819 -91 2034 -91 \nQ 1250 -91 836 529 \nQ 422 1150 422 2328 \nQ 422 3509 836 4129 \nQ 1250 4750 2034 4750 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-30\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <path d=\"M 106.200122 226.378306 \nL 106.200122 8.938306 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_2\">\n      <!-- 2000 -->\n      <g style=\"fill: #262626\" transform=\"translate(93.475122 240.976743)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \nL 3431 531 \nL 3431 0 \nL 469 0 \nL 469 531 \nQ 828 903 1448 1529 \nQ 2069 2156 2228 2338 \nQ 2531 2678 2651 2914 \nQ 2772 3150 2772 3378 \nQ 2772 3750 2511 3984 \nQ 2250 4219 1831 4219 \nQ 1534 4219 1204 4116 \nQ 875 4013 500 3803 \nL 500 4441 \nQ 881 4594 1212 4672 \nQ 1544 4750 1819 4750 \nQ 2544 4750 2975 4387 \nQ 3406 4025 3406 3419 \nQ 3406 3131 3298 2873 \nQ 3191 2616 2906 2266 \nQ 2828 2175 2409 1742 \nQ 1991 1309 1228 531 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-32\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <path d=\"M 167.078937 226.378306 \nL 167.078937 8.938306 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_3\">\n      <!-- 4000 -->\n      <g style=\"fill: #262626\" transform=\"translate(154.353937 240.976743)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \nL 825 1625 \nL 2419 1625 \nL 2419 4116 \nz\nM 2253 4666 \nL 3047 4666 \nL 3047 1625 \nL 3713 1625 \nL 3713 1100 \nL 3047 1100 \nL 3047 0 \nL 2419 0 \nL 2419 1100 \nL 313 1100 \nL 313 1709 \nL 2253 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-34\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <path d=\"M 227.957752 226.378306 \nL 227.957752 8.938306 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_4\">\n      <!-- 6000 -->\n      <g style=\"fill: #262626\" transform=\"translate(215.232752 240.976743)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-36\" d=\"M 2113 2584 \nQ 1688 2584 1439 2293 \nQ 1191 2003 1191 1497 \nQ 1191 994 1439 701 \nQ 1688 409 2113 409 \nQ 2538 409 2786 701 \nQ 3034 994 3034 1497 \nQ 3034 2003 2786 2293 \nQ 2538 2584 2113 2584 \nz\nM 3366 4563 \nL 3366 3988 \nQ 3128 4100 2886 4159 \nQ 2644 4219 2406 4219 \nQ 1781 4219 1451 3797 \nQ 1122 3375 1075 2522 \nQ 1259 2794 1537 2939 \nQ 1816 3084 2150 3084 \nQ 2853 3084 3261 2657 \nQ 3669 2231 3669 1497 \nQ 3669 778 3244 343 \nQ 2819 -91 2113 -91 \nQ 1303 -91 875 529 \nQ 447 1150 447 2328 \nQ 447 3434 972 4092 \nQ 1497 4750 2381 4750 \nQ 2619 4750 2861 4703 \nQ 3103 4656 3366 4563 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-36\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <path d=\"M 288.836567 226.378306 \nL 288.836567 8.938306 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_5\">\n      <!-- 8000 -->\n      <g style=\"fill: #262626\" transform=\"translate(276.111567 240.976743)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-38\" d=\"M 2034 2216 \nQ 1584 2216 1326 1975 \nQ 1069 1734 1069 1313 \nQ 1069 891 1326 650 \nQ 1584 409 2034 409 \nQ 2484 409 2743 651 \nQ 3003 894 3003 1313 \nQ 3003 1734 2745 1975 \nQ 2488 2216 2034 2216 \nz\nM 1403 2484 \nQ 997 2584 770 2862 \nQ 544 3141 544 3541 \nQ 544 4100 942 4425 \nQ 1341 4750 2034 4750 \nQ 2731 4750 3128 4425 \nQ 3525 4100 3525 3541 \nQ 3525 3141 3298 2862 \nQ 3072 2584 2669 2484 \nQ 3125 2378 3379 2068 \nQ 3634 1759 3634 1313 \nQ 3634 634 3220 271 \nQ 2806 -91 2034 -91 \nQ 1263 -91 848 271 \nQ 434 634 434 1313 \nQ 434 1759 690 2068 \nQ 947 2378 1403 2484 \nz\nM 1172 3481 \nQ 1172 3119 1398 2916 \nQ 1625 2713 2034 2713 \nQ 2441 2713 2670 2916 \nQ 2900 3119 2900 3481 \nQ 2900 3844 2670 4047 \nQ 2441 4250 2034 4250 \nQ 1625 4250 1398 4047 \nQ 1172 3844 1172 3481 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-38\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_6\">\n     <g id=\"line2d_6\">\n      <path d=\"M 349.715383 226.378306 \nL 349.715383 8.938306 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_6\">\n      <!-- 10000 -->\n      <g style=\"fill: #262626\" transform=\"translate(333.809133 240.976743)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-31\" d=\"M 794 531 \nL 1825 531 \nL 1825 4091 \nL 703 3866 \nL 703 4441 \nL 1819 4666 \nL 2450 4666 \nL 2450 531 \nL 3481 531 \nL 3481 0 \nL 794 0 \nL 794 531 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-31\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"254.492188\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_7\">\n      <path d=\"M 30.103125 197.813265 \nL 364.903125 197.813265 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.3 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 201.612483)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \nL 1344 794 \nL 1344 0 \nL 684 0 \nL 684 794 \nz\n\" transform=\"scale(0.015625)\"/>\n        <path id=\"DejaVuSans-33\" d=\"M 2597 2516 \nQ 3050 2419 3304 2112 \nQ 3559 1806 3559 1356 \nQ 3559 666 3084 287 \nQ 2609 -91 1734 -91 \nQ 1441 -91 1130 -33 \nQ 819 25 488 141 \nL 488 750 \nQ 750 597 1062 519 \nQ 1375 441 1716 441 \nQ 2309 441 2620 675 \nQ 2931 909 2931 1356 \nQ 2931 1769 2642 2001 \nQ 2353 2234 1838 2234 \nL 1294 2234 \nL 1294 2753 \nL 1863 2753 \nQ 2328 2753 2575 2939 \nQ 2822 3125 2822 3475 \nQ 2822 3834 2567 4026 \nQ 2313 4219 1838 4219 \nQ 1578 4219 1281 4162 \nQ 984 4106 628 3988 \nL 628 4550 \nQ 988 4650 1302 4700 \nQ 1616 4750 1894 4750 \nQ 2613 4750 3031 4423 \nQ 3450 4097 3450 3541 \nQ 3450 3153 3228 2886 \nQ 3006 2619 2597 2516 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-33\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_8\">\n      <path d=\"M 30.103125 160.450456 \nL 364.903125 160.450456 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0.4 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 164.249674)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-34\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_9\">\n      <path d=\"M 30.103125 123.087646 \nL 364.903125 123.087646 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_9\">\n      <!-- 0.5 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 126.886865)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-35\" d=\"M 691 4666 \nL 3169 4666 \nL 3169 4134 \nL 1269 4134 \nL 1269 2991 \nQ 1406 3038 1543 3061 \nQ 1681 3084 1819 3084 \nQ 2600 3084 3056 2656 \nQ 3513 2228 3513 1497 \nQ 3513 744 3044 326 \nQ 2575 -91 1722 -91 \nQ 1428 -91 1123 -41 \nQ 819 9 494 109 \nL 494 744 \nQ 775 591 1075 516 \nQ 1375 441 1709 441 \nQ 2250 441 2565 725 \nQ 2881 1009 2881 1497 \nQ 2881 1984 2565 2268 \nQ 2250 2553 1709 2553 \nQ 1456 2553 1204 2497 \nQ 953 2441 691 2322 \nL 691 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-35\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_10\">\n      <path d=\"M 30.103125 85.724837 \nL 364.903125 85.724837 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_10\">\n      <!-- 0.6 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 89.524056)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-36\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_11\">\n      <path d=\"M 30.103125 48.362028 \nL 364.903125 48.362028 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0.7 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 52.161247)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-37\" d=\"M 525 4666 \nL 3525 4666 \nL 3525 4397 \nL 1831 0 \nL 1172 0 \nL 2766 4134 \nL 525 4134 \nL 525 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-37\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_12\">\n      <path d=\"M 30.103125 10.999219 \nL 364.903125 10.999219 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_12\">\n      <!-- 0.8 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 14.798437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-38\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_13\">\n    <path d=\"M 45.321307 18.821942 \nL 45.68658 31.58671 \nL 46.082292 38.127066 \nL 46.508444 41.57762 \nL 46.995474 43.747249 \nL 47.604262 45.45275 \nL 48.365248 46.921321 \nL 49.217551 48.085328 \nL 50.130733 48.959179 \nL 51.135234 49.613983 \nL 52.291931 50.102742 \nL 53.753023 50.468193 \nL 55.731584 50.717705 \nL 58.866843 50.863908 \nL 65.928786 50.918625 \nL 144.462457 50.922433 \nL 349.684943 50.922433 \nL 349.684943 50.922433 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: round\"/>\n   </g>\n   <g id=\"line2d_14\">\n    <path d=\"M 45.321307 29.680623 \nL 47.512944 29.680623 \nL 47.634702 123.087646 \nL 95.455011 123.087646 \nL 95.576769 216.494669 \nL 143.30576 216.494669 \nL 143.427517 123.087646 \nL 143.549275 216.494669 \nL 143.579714 216.494669 \nL 143.701472 123.087646 \nL 143.82323 216.494669 \nL 143.853669 216.494669 \nL 143.975427 123.087646 \nL 144.097184 216.494669 \nL 144.127624 216.494669 \nL 144.249381 123.087646 \nL 144.371139 216.494669 \nL 144.401578 216.494669 \nL 144.523336 123.087646 \nL 144.645094 216.494669 \nL 144.675533 216.494669 \nL 144.797291 123.087646 \nL 144.919048 216.494669 \nL 144.949488 216.494669 \nL 145.071245 123.087646 \nL 145.193003 216.494669 \nL 145.314761 123.087646 \nL 145.436518 216.494669 \nL 145.558276 123.087646 \nL 145.588715 123.087646 \nL 145.710473 216.494669 \nL 145.832231 123.087646 \nL 145.86267 123.087646 \nL 145.984428 216.494669 \nL 146.106185 123.087646 \nL 146.136625 123.087646 \nL 146.258382 216.494669 \nL 146.38014 123.087646 \nL 146.410579 123.087646 \nL 146.532337 216.494669 \nL 146.654095 123.087646 \nL 146.684534 123.087646 \nL 146.806292 216.494669 \nL 146.928049 123.087646 \nL 146.958489 123.087646 \nL 147.080246 216.494669 \nL 147.202004 123.087646 \nL 147.232443 123.087646 \nL 147.354201 216.494669 \nL 147.475959 123.087646 \nL 147.506398 123.087646 \nL 147.628156 216.494669 \nL 147.749913 123.087646 \nL 147.780353 123.087646 \nL 147.90211 216.494669 \nL 148.023868 123.087646 \nL 148.054307 123.087646 \nL 148.176065 216.494669 \nL 148.297823 123.087646 \nL 148.328262 123.087646 \nL 148.45002 216.494669 \nL 148.571777 123.087646 \nL 148.602217 123.087646 \nL 148.723974 216.494669 \nL 148.845732 123.087646 \nL 349.684943 123.087646 \nL 349.684943 123.087646 \n\" clip-path=\"url(#pac3db0900a)\" style=\"fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: round\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 30.103125 226.378306 \nL 30.103125 8.938306 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 364.903125 226.378306 \nL 364.903125 8.938306 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 30.103125 226.378306 \nL 364.903125 226.378306 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 30.103125 8.938306 \nL 364.903125 8.938306 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 303.96875 46.294556 \nL 357.903125 46.294556 \nQ 359.903125 46.294556 359.903125 44.294556 \nL 359.903125 15.938306 \nQ 359.903125 13.938306 357.903125 13.938306 \nL 303.96875 13.938306 \nQ 301.96875 13.938306 301.96875 15.938306 \nL 301.96875 44.294556 \nQ 301.96875 46.294556 303.96875 46.294556 \nz\n\" style=\"fill: #eaeaf2; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter\"/>\n    </g>\n    <g id=\"line2d_15\">\n     <path d=\"M 305.96875 22.036743 \nL 315.96875 22.036743 \nL 325.96875 22.036743 \n\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: round\"/>\n    </g>\n    <g id=\"text_13\">\n     <!-- Loss -->\n     <g style=\"fill: #262626\" transform=\"translate(333.96875 25.536743)scale(0.1 -0.1)\">\n      <defs>\n       <path id=\"DejaVuSans-4c\" d=\"M 628 4666 \nL 1259 4666 \nL 1259 531 \nL 3531 531 \nL 3531 0 \nL 628 0 \nL 628 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-6f\" d=\"M 1959 3097 \nQ 1497 3097 1228 2736 \nQ 959 2375 959 1747 \nQ 959 1119 1226 758 \nQ 1494 397 1959 397 \nQ 2419 397 2687 759 \nQ 2956 1122 2956 1747 \nQ 2956 2369 2687 2733 \nQ 2419 3097 1959 3097 \nz\nM 1959 3584 \nQ 2709 3584 3137 3096 \nQ 3566 2609 3566 1747 \nQ 3566 888 3137 398 \nQ 2709 -91 1959 -91 \nQ 1206 -91 779 398 \nQ 353 888 353 1747 \nQ 353 2609 779 3096 \nQ 1206 3584 1959 3584 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \nL 2834 2853 \nQ 2591 2978 2328 3040 \nQ 2066 3103 1784 3103 \nQ 1356 3103 1142 2972 \nQ 928 2841 928 2578 \nQ 928 2378 1081 2264 \nQ 1234 2150 1697 2047 \nL 1894 2003 \nQ 2506 1872 2764 1633 \nQ 3022 1394 3022 966 \nQ 3022 478 2636 193 \nQ 2250 -91 1575 -91 \nQ 1294 -91 989 -36 \nQ 684 19 347 128 \nL 347 722 \nQ 666 556 975 473 \nQ 1284 391 1588 391 \nQ 1994 391 2212 530 \nQ 2431 669 2431 922 \nQ 2431 1156 2273 1281 \nQ 2116 1406 1581 1522 \nL 1381 1569 \nQ 847 1681 609 1914 \nQ 372 2147 372 2553 \nQ 372 3047 722 3315 \nQ 1072 3584 1716 3584 \nQ 2034 3584 2315 3537 \nQ 2597 3491 2834 3397 \nz\n\" transform=\"scale(0.015625)\"/>\n      </defs>\n      <use xlink:href=\"#DejaVuSans-4c\"/>\n      <use xlink:href=\"#DejaVuSans-6f\" x=\"53.962891\"/>\n      <use xlink:href=\"#DejaVuSans-73\" x=\"115.144531\"/>\n      <use xlink:href=\"#DejaVuSans-73\" x=\"167.244141\"/>\n     </g>\n    </g>\n    <g id=\"line2d_16\">\n     <path d=\"M 305.96875 36.714868 \nL 315.96875 36.714868 \nL 325.96875 36.714868 \n\" style=\"fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: round\"/>\n    </g>\n    <g id=\"text_14\">\n     <!-- Acc -->\n     <g style=\"fill: #262626\" transform=\"translate(333.96875 40.214868)scale(0.1 -0.1)\">\n      <defs>\n       <path id=\"DejaVuSans-41\" d=\"M 2188 4044 \nL 1331 1722 \nL 3047 1722 \nL 2188 4044 \nz\nM 1831 4666 \nL 2547 4666 \nL 4325 0 \nL 3669 0 \nL 3244 1197 \nL 1141 1197 \nL 716 0 \nL 50 0 \nL 1831 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-63\" d=\"M 3122 3366 \nL 3122 2828 \nQ 2878 2963 2633 3030 \nQ 2388 3097 2138 3097 \nQ 1578 3097 1268 2742 \nQ 959 2388 959 1747 \nQ 959 1106 1268 751 \nQ 1578 397 2138 397 \nQ 2388 397 2633 464 \nQ 2878 531 3122 666 \nL 3122 134 \nQ 2881 22 2623 -34 \nQ 2366 -91 2075 -91 \nQ 1284 -91 818 406 \nQ 353 903 353 1747 \nQ 353 2603 823 3093 \nQ 1294 3584 2113 3584 \nQ 2378 3584 2631 3529 \nQ 2884 3475 3122 3366 \nz\n\" transform=\"scale(0.015625)\"/>\n      </defs>\n      <use xlink:href=\"#DejaVuSans-41\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"66.658203\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"121.638672\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"pac3db0900a\">\n   <rect x=\"30.103125\" y=\"8.938306\" width=\"334.8\" height=\"217.44\"/>\n  </clipPath>\n </defs>\n</svg>\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_record)\n",
    "plt.plot(acc_record)\n",
    "plt.legend([\"Loss\", \"Acc\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "        self.W1 = nn.Parameter(torch.randn((2, 2)))\n",
    "        self.b1 = nn.Parameter(torch.randn((2,)))\n",
    "        self.W2 = nn.Parameter(torch.randn((2, 1)))\n",
    "        self.b2 = nn.Parameter(torch.randn((1,)))\n",
    "\n",
    "    def forward(self, X: torch.Tensor) -> torch.Tensor:\n",
    "        hidden = torch.sigmoid(X @ self.W1 + self.b1)\n",
    "        hypothesis = torch.sigmoid(hidden @ self.W2 + self.b2)\n",
    "        return hypothesis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step=0, loss=0.726954460144043, accuracy=0.5\n",
      "Hypothesis:  tensor([[0.4806],\n",
      "        [0.3830],\n",
      "        [0.4241],\n",
      "        [0.3527]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=2000, loss=0.3294403851032257, accuracy=1.0\n",
      "Hypothesis:  tensor([[0.1842],\n",
      "        [0.7058],\n",
      "        [0.7084],\n",
      "        [0.3436]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=4000, loss=0.061377570033073425, accuracy=1.0\n",
      "Hypothesis:  tensor([[0.0454],\n",
      "        [0.9356],\n",
      "        [0.9356],\n",
      "        [0.0638]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=6000, loss=0.029097670689225197, accuracy=1.0\n",
      "Hypothesis:  tensor([[0.0231],\n",
      "        [0.9688],\n",
      "        [0.9688],\n",
      "        [0.0292]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n",
      "step=8000, loss=0.018635760992765427, accuracy=1.0\n",
      "Hypothesis:  tensor([[0.0152],\n",
      "        [0.9799],\n",
      "        [0.9799],\n",
      "        [0.0184]], grad_fn=<SigmoidBackward0>)\n",
      "Correct:  tensor([[0],\n",
      "        [1],\n",
      "        [1],\n",
      "        [0]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fc8686cef70>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"372.821633pt\" height=\"248.518125pt\" viewBox=\"0 0 372.821633 248.518125\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n <metadata>\n  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n   <cc:Work>\n    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n    <dc:date>2022-07-13T09:37:06.378266</dc:date>\n    <dc:format>image/svg+xml</dc:format>\n    <dc:creator>\n     <cc:Agent>\n      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n     </cc:Agent>\n    </dc:creator>\n   </cc:Work>\n  </rdf:RDF>\n </metadata>\n <defs>\n  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 248.518125 \nL 372.821633 248.518125 \nL 372.821633 0 \nL 0 0 \nz\n\" style=\"fill: #ffffff\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 30.103125 224.64 \nL 364.903125 224.64 \nL 364.903125 7.2 \nL 30.103125 7.2 \nz\n\" style=\"fill: #eaeaf2\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <path d=\"M 45.321307 224.64 \nL 45.321307 7.2 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <g style=\"fill: #262626\" transform=\"translate(42.140057 239.238438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \nQ 1547 4250 1301 3770 \nQ 1056 3291 1056 2328 \nQ 1056 1369 1301 889 \nQ 1547 409 2034 409 \nQ 2525 409 2770 889 \nQ 3016 1369 3016 2328 \nQ 3016 3291 2770 3770 \nQ 2525 4250 2034 4250 \nz\nM 2034 4750 \nQ 2819 4750 3233 4129 \nQ 3647 3509 3647 2328 \nQ 3647 1150 3233 529 \nQ 2819 -91 2034 -91 \nQ 1250 -91 836 529 \nQ 422 1150 422 2328 \nQ 422 3509 836 4129 \nQ 1250 4750 2034 4750 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-30\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <path d=\"M 106.200122 224.64 \nL 106.200122 7.2 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_2\">\n      <!-- 2000 -->\n      <g style=\"fill: #262626\" transform=\"translate(93.475122 239.238438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \nL 3431 531 \nL 3431 0 \nL 469 0 \nL 469 531 \nQ 828 903 1448 1529 \nQ 2069 2156 2228 2338 \nQ 2531 2678 2651 2914 \nQ 2772 3150 2772 3378 \nQ 2772 3750 2511 3984 \nQ 2250 4219 1831 4219 \nQ 1534 4219 1204 4116 \nQ 875 4013 500 3803 \nL 500 4441 \nQ 881 4594 1212 4672 \nQ 1544 4750 1819 4750 \nQ 2544 4750 2975 4387 \nQ 3406 4025 3406 3419 \nQ 3406 3131 3298 2873 \nQ 3191 2616 2906 2266 \nQ 2828 2175 2409 1742 \nQ 1991 1309 1228 531 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-32\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <path d=\"M 167.078937 224.64 \nL 167.078937 7.2 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_3\">\n      <!-- 4000 -->\n      <g style=\"fill: #262626\" transform=\"translate(154.353937 239.238438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \nL 825 1625 \nL 2419 1625 \nL 2419 4116 \nz\nM 2253 4666 \nL 3047 4666 \nL 3047 1625 \nL 3713 1625 \nL 3713 1100 \nL 3047 1100 \nL 3047 0 \nL 2419 0 \nL 2419 1100 \nL 313 1100 \nL 313 1709 \nL 2253 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-34\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <path d=\"M 227.957752 224.64 \nL 227.957752 7.2 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_4\">\n      <!-- 6000 -->\n      <g style=\"fill: #262626\" transform=\"translate(215.232752 239.238438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-36\" d=\"M 2113 2584 \nQ 1688 2584 1439 2293 \nQ 1191 2003 1191 1497 \nQ 1191 994 1439 701 \nQ 1688 409 2113 409 \nQ 2538 409 2786 701 \nQ 3034 994 3034 1497 \nQ 3034 2003 2786 2293 \nQ 2538 2584 2113 2584 \nz\nM 3366 4563 \nL 3366 3988 \nQ 3128 4100 2886 4159 \nQ 2644 4219 2406 4219 \nQ 1781 4219 1451 3797 \nQ 1122 3375 1075 2522 \nQ 1259 2794 1537 2939 \nQ 1816 3084 2150 3084 \nQ 2853 3084 3261 2657 \nQ 3669 2231 3669 1497 \nQ 3669 778 3244 343 \nQ 2819 -91 2113 -91 \nQ 1303 -91 875 529 \nQ 447 1150 447 2328 \nQ 447 3434 972 4092 \nQ 1497 4750 2381 4750 \nQ 2619 4750 2861 4703 \nQ 3103 4656 3366 4563 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-36\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <path d=\"M 288.836567 224.64 \nL 288.836567 7.2 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_5\">\n      <!-- 8000 -->\n      <g style=\"fill: #262626\" transform=\"translate(276.111567 239.238438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-38\" d=\"M 2034 2216 \nQ 1584 2216 1326 1975 \nQ 1069 1734 1069 1313 \nQ 1069 891 1326 650 \nQ 1584 409 2034 409 \nQ 2484 409 2743 651 \nQ 3003 894 3003 1313 \nQ 3003 1734 2745 1975 \nQ 2488 2216 2034 2216 \nz\nM 1403 2484 \nQ 997 2584 770 2862 \nQ 544 3141 544 3541 \nQ 544 4100 942 4425 \nQ 1341 4750 2034 4750 \nQ 2731 4750 3128 4425 \nQ 3525 4100 3525 3541 \nQ 3525 3141 3298 2862 \nQ 3072 2584 2669 2484 \nQ 3125 2378 3379 2068 \nQ 3634 1759 3634 1313 \nQ 3634 634 3220 271 \nQ 2806 -91 2034 -91 \nQ 1263 -91 848 271 \nQ 434 634 434 1313 \nQ 434 1759 690 2068 \nQ 947 2378 1403 2484 \nz\nM 1172 3481 \nQ 1172 3119 1398 2916 \nQ 1625 2713 2034 2713 \nQ 2441 2713 2670 2916 \nQ 2900 3119 2900 3481 \nQ 2900 3844 2670 4047 \nQ 2441 4250 2034 4250 \nQ 1625 4250 1398 4047 \nQ 1172 3844 1172 3481 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-38\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_6\">\n     <g id=\"line2d_6\">\n      <path d=\"M 349.715383 224.64 \nL 349.715383 7.2 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_6\">\n      <!-- 10000 -->\n      <g style=\"fill: #262626\" transform=\"translate(333.809133 239.238438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-31\" d=\"M 794 531 \nL 1825 531 \nL 1825 4091 \nL 703 3866 \nL 703 4441 \nL 1819 4666 \nL 2450 4666 \nL 2450 531 \nL 3481 531 \nL 3481 0 \nL 794 0 \nL 794 531 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-31\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"127.246094\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"190.869141\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"254.492188\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_7\">\n      <path d=\"M 30.103125 217.482569 \nL 364.903125 217.482569 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.0 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 221.281788)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \nL 1344 794 \nL 1344 0 \nL 684 0 \nL 684 794 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_8\">\n      <path d=\"M 30.103125 177.402782 \nL 364.903125 177.402782 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0.2 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 181.202001)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-32\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_9\">\n      <path d=\"M 30.103125 137.322996 \nL 364.903125 137.322996 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_9\">\n      <!-- 0.4 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 141.122215)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-34\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_10\">\n      <path d=\"M 30.103125 97.243209 \nL 364.903125 97.243209 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_10\">\n      <!-- 0.6 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 101.042428)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-36\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_11\">\n      <path d=\"M 30.103125 57.163423 \nL 364.903125 57.163423 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0.8 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 60.962642)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-38\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_12\">\n      <path d=\"M 30.103125 17.083636 \nL 364.903125 17.083636 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linecap: round\"/>\n     </g>\n     <g id=\"text_12\">\n      <!-- 1.0 -->\n      <g style=\"fill: #262626\" transform=\"translate(7.2 20.882855)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-31\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_13\">\n    <path d=\"M 45.321307 71.801671 \nL 45.68658 74.639435 \nL 46.051853 75.906052 \nL 46.478004 76.616702 \nL 47.056353 77.127625 \nL 48.121732 77.721874 \nL 50.069854 78.537363 \nL 52.83984 79.441781 \nL 62.367375 82.364304 \nL 65.19824 83.56636 \nL 67.724711 84.878629 \nL 70.068545 86.344089 \nL 72.321061 88.017304 \nL 74.482259 89.901785 \nL 76.613018 92.05827 \nL 78.743776 94.537119 \nL 80.904974 97.40011 \nL 83.127051 100.720596 \nL 85.470885 104.636644 \nL 87.936477 109.199534 \nL 90.615145 114.636132 \nL 93.628647 121.27457 \nL 97.311815 129.958659 \nL 108.117805 155.808522 \nL 111.100867 162.143612 \nL 113.809974 167.382095 \nL 116.366884 171.850001 \nL 118.862916 175.769719 \nL 121.328508 179.234249 \nL 123.824539 182.360519 \nL 126.35101 185.17164 \nL 128.968799 187.750682 \nL 131.677906 190.105778 \nL 134.508771 192.27075 \nL 137.491833 194.271136 \nL 140.657531 196.125827 \nL 144.066745 197.86319 \nL 147.719474 199.475175 \nL 151.707036 200.991608 \nL 156.090311 202.419132 \nL 160.991056 203.775616 \nL 166.43971 205.047215 \nL 172.527591 206.236408 \nL 179.437337 207.356452 \nL 187.412461 208.417475 \nL 196.635602 209.412448 \nL 207.411152 210.343141 \nL 220.165264 211.212577 \nL 235.506725 212.024245 \nL 253.922567 212.767488 \nL 276.447729 213.448525 \nL 304.421544 214.068773 \nL 339.426863 214.625228 \nL 349.684943 214.756364 \nL 349.684943 214.756364 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: round\"/>\n   </g>\n   <g id=\"line2d_14\">\n    <path d=\"M 45.321307 117.283103 \nL 45.503943 117.283103 \nL 45.625701 167.382836 \nL 46.417125 167.382836 \nL 46.538883 117.283103 \nL 58.714646 117.283103 \nL 58.836404 67.18337 \nL 69.611954 67.18337 \nL 69.733712 17.083636 \nL 349.684943 17.083636 \nL 349.684943 17.083636 \n\" clip-path=\"url(#p48a3ab7a4c)\" style=\"fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: round\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 30.103125 224.64 \nL 30.103125 7.2 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 364.903125 224.64 \nL 364.903125 7.2 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 30.103125 224.64 \nL 364.903125 224.64 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 30.103125 7.2 \nL 364.903125 7.2 \n\" style=\"fill: none; stroke: #ffffff; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 37.103125 219.64 \nL 91.0375 219.64 \nQ 93.0375 219.64 93.0375 217.64 \nL 93.0375 189.28375 \nQ 93.0375 187.28375 91.0375 187.28375 \nL 37.103125 187.28375 \nQ 35.103125 187.28375 35.103125 189.28375 \nL 35.103125 217.64 \nQ 35.103125 219.64 37.103125 219.64 \nz\n\" style=\"fill: #eaeaf2; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter\"/>\n    </g>\n    <g id=\"line2d_15\">\n     <path d=\"M 39.103125 195.382187 \nL 49.103125 195.382187 \nL 59.103125 195.382187 \n\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: round\"/>\n    </g>\n    <g id=\"text_13\">\n     <!-- Loss -->\n     <g style=\"fill: #262626\" transform=\"translate(67.103125 198.882187)scale(0.1 -0.1)\">\n      <defs>\n       <path id=\"DejaVuSans-4c\" d=\"M 628 4666 \nL 1259 4666 \nL 1259 531 \nL 3531 531 \nL 3531 0 \nL 628 0 \nL 628 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-6f\" d=\"M 1959 3097 \nQ 1497 3097 1228 2736 \nQ 959 2375 959 1747 \nQ 959 1119 1226 758 \nQ 1494 397 1959 397 \nQ 2419 397 2687 759 \nQ 2956 1122 2956 1747 \nQ 2956 2369 2687 2733 \nQ 2419 3097 1959 3097 \nz\nM 1959 3584 \nQ 2709 3584 3137 3096 \nQ 3566 2609 3566 1747 \nQ 3566 888 3137 398 \nQ 2709 -91 1959 -91 \nQ 1206 -91 779 398 \nQ 353 888 353 1747 \nQ 353 2609 779 3096 \nQ 1206 3584 1959 3584 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \nL 2834 2853 \nQ 2591 2978 2328 3040 \nQ 2066 3103 1784 3103 \nQ 1356 3103 1142 2972 \nQ 928 2841 928 2578 \nQ 928 2378 1081 2264 \nQ 1234 2150 1697 2047 \nL 1894 2003 \nQ 2506 1872 2764 1633 \nQ 3022 1394 3022 966 \nQ 3022 478 2636 193 \nQ 2250 -91 1575 -91 \nQ 1294 -91 989 -36 \nQ 684 19 347 128 \nL 347 722 \nQ 666 556 975 473 \nQ 1284 391 1588 391 \nQ 1994 391 2212 530 \nQ 2431 669 2431 922 \nQ 2431 1156 2273 1281 \nQ 2116 1406 1581 1522 \nL 1381 1569 \nQ 847 1681 609 1914 \nQ 372 2147 372 2553 \nQ 372 3047 722 3315 \nQ 1072 3584 1716 3584 \nQ 2034 3584 2315 3537 \nQ 2597 3491 2834 3397 \nz\n\" transform=\"scale(0.015625)\"/>\n      </defs>\n      <use xlink:href=\"#DejaVuSans-4c\"/>\n      <use xlink:href=\"#DejaVuSans-6f\" x=\"53.962891\"/>\n      <use xlink:href=\"#DejaVuSans-73\" x=\"115.144531\"/>\n      <use xlink:href=\"#DejaVuSans-73\" x=\"167.244141\"/>\n     </g>\n    </g>\n    <g id=\"line2d_16\">\n     <path d=\"M 39.103125 210.060313 \nL 49.103125 210.060313 \nL 59.103125 210.060313 \n\" style=\"fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: round\"/>\n    </g>\n    <g id=\"text_14\">\n     <!-- Acc -->\n     <g style=\"fill: #262626\" transform=\"translate(67.103125 213.560313)scale(0.1 -0.1)\">\n      <defs>\n       <path id=\"DejaVuSans-41\" d=\"M 2188 4044 \nL 1331 1722 \nL 3047 1722 \nL 2188 4044 \nz\nM 1831 4666 \nL 2547 4666 \nL 4325 0 \nL 3669 0 \nL 3244 1197 \nL 1141 1197 \nL 716 0 \nL 50 0 \nL 1831 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-63\" d=\"M 3122 3366 \nL 3122 2828 \nQ 2878 2963 2633 3030 \nQ 2388 3097 2138 3097 \nQ 1578 3097 1268 2742 \nQ 959 2388 959 1747 \nQ 959 1106 1268 751 \nQ 1578 397 2138 397 \nQ 2388 397 2633 464 \nQ 2878 531 3122 666 \nL 3122 134 \nQ 2881 22 2623 -34 \nQ 2366 -91 2075 -91 \nQ 1284 -91 818 406 \nQ 353 903 353 1747 \nQ 353 2603 823 3093 \nQ 1294 3584 2113 3584 \nQ 2378 3584 2631 3529 \nQ 2884 3475 3122 3366 \nz\n\" transform=\"scale(0.015625)\"/>\n      </defs>\n      <use xlink:href=\"#DejaVuSans-41\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"66.658203\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"121.638672\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p48a3ab7a4c\">\n   <rect x=\"30.103125\" y=\"7.2\" width=\"334.8\" height=\"217.44\"/>\n  </clipPath>\n </defs>\n</svg>\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = Network()\n",
    "cost = Cost()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "model.to(device)\n",
    "cost.to(device)\n",
    "x_data = x_data.to(device)\n",
    "y_data = y_data.to(device)\n",
    "\n",
    "loss_record = []\n",
    "acc_record = []\n",
    "\n",
    "for step in range(10000):\n",
    "    hypot = model(x_data)\n",
    "    loss = cost(hypot, y_data)\n",
    "    loss_record.append(float(loss))\n",
    "    acc = accuracy(hypot, y_data)\n",
    "    acc_record.append(float(acc))\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if step % 2000 == 0:\n",
    "\n",
    "        print(f\"step={step}, loss={float(loss)}, accuracy={float(acc)}\")\n",
    "        print(\"Hypothesis: \", hypot)\n",
    "        print(\"Correct: \", y_data)\n",
    "\n",
    "plt.plot(loss_record)\n",
    "plt.plot(acc_record)\n",
    "plt.legend([\"Loss\", \"Acc\"])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "writer = SummaryWriter()\n",
    "writer.add_graph(Network(), x_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![graph](./graph.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Layer(nn.Module):\n",
    "    def __init__(self, in_features, out_features, activate):\n",
    "        super().__init__()\n",
    "        self.W = nn.Parameter(torch.empty((in_features, out_features)))\n",
    "        nn.init.normal_(self.W, -1.0, 1.0)\n",
    "        self.b = nn.Parameter(torch.zeros(out_features))\n",
    "        self.act = activate\n",
    "    def forward(self, X: torch.Tensor) -> torch.Tensor:\n",
    "        return torch.nn.functional.relu(X @ self.W + self.b);\n",
    "\n",
    "class Network10(nn.Module):\n",
    "    def __init__(self, activate) -> None:\n",
    "        super().__init__()\n",
    "        self.layers = nn.Sequential(\n",
    "            Layer(2, 5, activate),\n",
    "            *([Layer(5, 5, activate)] * 9),\n",
    "            Layer(5, 1, activate)\n",
    "        )\n",
    "\n",
    "    def forward(self, X: torch.Tensor) -> torch.Tensor:\n",
    "        return self.layers(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Network10(torch.sigmoid)\n",
    "cost = Cost()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "device = torch.device(\"cuda:0\")\n",
    "\n",
    "model.to(device)\n",
    "cost.to(device)\n",
    "x_data = x_data.to(device)\n",
    "y_data = y_data.to(device)\n",
    "\n",
    "for step in range(10000):\n",
    "    hypot = model(x_data)\n",
    "    loss = cost(hypot, y_data)\n",
    "    writer.add_scalar(\"Network10/loss\", loss, step)\n",
    "    acc = accuracy(hypot, y_data)\n",
    "    writer.add_scalar(\"Network10/acc\", float(acc), step)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n",
      "tensor([[0.],\n",
      "        [0.],\n",
      "        [0.],\n",
      "        [0.]], device='cuda:0', grad_fn=<ReluBackward0>)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/guoruiming/workspace/pyla/torch/reignited/xor_logistic_regression.ipynb Cell 12\u001b[0m in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bmaker-jupyter/home/guoruiming/workspace/pyla/torch/reignited/xor_logistic_regression.ipynb#ch0000013vscode-remote?line=17'>18</a>\u001b[0m optimizer\u001b[39m.\u001b[39mzero_grad()\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bmaker-jupyter/home/guoruiming/workspace/pyla/torch/reignited/xor_logistic_regression.ipynb#ch0000013vscode-remote?line=18'>19</a>\u001b[0m loss\u001b[39m.\u001b[39mbackward()\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bmaker-jupyter/home/guoruiming/workspace/pyla/torch/reignited/xor_logistic_regression.ipynb#ch0000013vscode-remote?line=19'>20</a>\u001b[0m optimizer\u001b[39m.\u001b[39;49mstep()\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/optim/optimizer.py:88\u001b[0m, in \u001b[0;36mOptimizer._hook_for_profile.<locals>.profile_hook_step.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m profile_name \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mOptimizer.step#\u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m.step\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(obj\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m)\n\u001b[1;32m     87\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mautograd\u001b[39m.\u001b[39mprofiler\u001b[39m.\u001b[39mrecord_function(profile_name):\n\u001b[0;32m---> 88\u001b[0m     \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py:27\u001b[0m, in \u001b[0;36m_DecoratorContextManager.__call__.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[39m@functools\u001b[39m\u001b[39m.\u001b[39mwraps(func)\n\u001b[1;32m     25\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdecorate_context\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m     26\u001b[0m     \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mclone():\n\u001b[0;32m---> 27\u001b[0m         \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/optim/sgd.py:144\u001b[0m, in \u001b[0;36mSGD.step\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    142\u001b[0m             momentum_buffer_list\u001b[39m.\u001b[39mappend(state[\u001b[39m'\u001b[39m\u001b[39mmomentum_buffer\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[0;32m--> 144\u001b[0m F\u001b[39m.\u001b[39;49msgd(params_with_grad,\n\u001b[1;32m    145\u001b[0m       d_p_list,\n\u001b[1;32m    146\u001b[0m       momentum_buffer_list,\n\u001b[1;32m    147\u001b[0m       weight_decay\u001b[39m=\u001b[39;49mweight_decay,\n\u001b[1;32m    148\u001b[0m       momentum\u001b[39m=\u001b[39;49mmomentum,\n\u001b[1;32m    149\u001b[0m       lr\u001b[39m=\u001b[39;49mlr,\n\u001b[1;32m    150\u001b[0m       dampening\u001b[39m=\u001b[39;49mdampening,\n\u001b[1;32m    151\u001b[0m       nesterov\u001b[39m=\u001b[39;49mnesterov,\n\u001b[1;32m    152\u001b[0m       maximize\u001b[39m=\u001b[39;49mmaximize,)\n\u001b[1;32m    154\u001b[0m \u001b[39m# update momentum_buffers in state\u001b[39;00m\n\u001b[1;32m    155\u001b[0m \u001b[39mfor\u001b[39;00m p, momentum_buffer \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(params_with_grad, momentum_buffer_list):\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/optim/_functional.py:194\u001b[0m, in \u001b[0;36msgd\u001b[0;34m(params, d_p_list, momentum_buffer_list, weight_decay, momentum, lr, dampening, nesterov, maximize)\u001b[0m\n\u001b[1;32m    191\u001b[0m         d_p \u001b[39m=\u001b[39m buf\n\u001b[1;32m    193\u001b[0m alpha \u001b[39m=\u001b[39m lr \u001b[39mif\u001b[39;00m maximize \u001b[39melse\u001b[39;00m \u001b[39m-\u001b[39mlr\n\u001b[0;32m--> 194\u001b[0m param\u001b[39m.\u001b[39;49madd_(d_p, alpha\u001b[39m=\u001b[39;49malpha)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = Network10(torch.nn.ReLU())\n",
    "cost = Cost()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "device = torch.device(\"cuda:0\")\n",
    "\n",
    "model.to(device)\n",
    "cost.to(device)\n",
    "x_data = x_data.to(device)\n",
    "y_data = y_data.to(device)\n",
    "\n",
    "for step in range(10000):\n",
    "    hypot = model(x_data)\n",
    "    print(hypot)\n",
    "    loss = cost(hypot, y_data)\n",
    "    writer.add_scalar(\"Network10-RELU/loss\", loss, step)\n",
    "    acc = accuracy(hypot, y_data)\n",
    "    writer.add_scalar(\"Network10-RELU/acc\", float(acc), step)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
